From 21f0255f9c03c403c4cd6803100c1ada436f3b33 Mon Sep 17 00:00:00 2001 From: "mwilli2@equilibrium.research.intel-research.net" Date: Tue, 8 Jun 2004 11:12:35 +0000 Subject: [PATCH] bitkeeper revision 1.934.2.2 (40c59f23b5OBWV0EcUOWaE-IXlUYNQ) Infrastructure for restartable block device drivers. --- tools/examples/xc_dom_create.py | 108 ++++++++++-------- tools/xend/lib/blkif.py | 68 ++++++++++- tools/xend/lib/main.py | 6 +- tools/xend/lib/manager.py | 10 ++ .../arch/xen/drivers/blkif/backend/main.c | 3 +- 5 files changed, 138 insertions(+), 57 deletions(-) diff --git a/tools/examples/xc_dom_create.py b/tools/examples/xc_dom_create.py index 610b63c851..79023f6c2e 100755 --- a/tools/examples/xc_dom_create.py +++ b/tools/examples/xc_dom_create.py @@ -273,66 +273,78 @@ def make_domain(): # set the expertise level appropriately xenctl.utils.VBD_EXPERT_MODE = vbd_expert - if new_io_world: - cmsg = 'new_block_interface(dom='+str(id)+')' - xend_response = xenctl.utils.xend_control_message(cmsg) - if not xend_response['success']: - print "Error creating block interface" - print "Error type: " + xend_response['error_type'] - if xend_response['error_type'] == 'exception': - print "Exception type: " + xend_response['exception_type'] - print "Exception val: " + xend_response['exception_value'] - xc.domain_destroy ( dom=id ) - sys.exit() - - for ( uname, virt_name, rw ) in vbd_list: - virt_dev = xenctl.utils.blkdev_name_to_number( virt_name ) - - segments = xenctl.utils.lookup_disk_uname( uname ) - if not segments: - print "Error looking up %s\n" % uname - xc.domain_destroy ( dom=id ) - sys.exit() - + if not (flags & 4): # It's not a block backend (or it's old IO world) if new_io_world: - if len(segments) > 1: - print "New I/O world cannot deal with multi-extent vdisks" - xc.domain_destroy ( dom=id ) - sys.exit() - seg = segments[0] - cmsg = 'new_block_device(dom=' + str(id) + \ - ',handle=0,vdev=' + str(virt_dev) + \ - ',pdev=' + str(seg['device']) + \ - ',start_sect=' + str(seg['start_sector']) + \ - ',nr_sect=' + str(seg['nr_sectors']) + \ - ',readonly=' + str(not re.match('w',rw)) + ')' + cmsg = 'new_block_interface(dom='+str(id)+')' xend_response = xenctl.utils.xend_control_message(cmsg) if not xend_response['success']: - print "Error creating virtual block device" + print "Error creating block interface" print "Error type: " + xend_response['error_type'] if xend_response['error_type'] == 'exception': print "Exception type: " + xend_response['exception_type'] print "Exception val: " + xend_response['exception_value'] xc.domain_destroy ( dom=id ) sys.exit() - else: - # check that setting up this VBD won't violate the sharing - # allowed by the current VBD expertise level - if xenctl.utils.vd_extents_validate(segments, - rw=='w' or rw=='rw') < 0: - xc.domain_destroy( dom = id ) - sys.exit() - - if xc.vbd_create( dom=id, vbd=virt_dev, - writeable= rw=='w' or rw=='rw' ): - print "Error creating VBD %d (writeable=%d)\n" % (virt_dev,rw) + + for ( uname, virt_name, rw ) in vbd_list: + virt_dev = xenctl.utils.blkdev_name_to_number( virt_name ) + + segments = xenctl.utils.lookup_disk_uname( uname ) + if not segments: + print "Error looking up %s\n" % uname xc.domain_destroy ( dom=id ) sys.exit() + + if new_io_world: + if len(segments) > 1: + print "New I/O world cannot deal with multi-extent vdisks" + xc.domain_destroy ( dom=id ) + sys.exit() + seg = segments[0] + cmsg = 'new_block_device(dom=' + str(id) + \ + ',handle=0,vdev=' + str(virt_dev) + \ + ',pdev=' + str(seg['device']) + \ + ',start_sect=' + str(seg['start_sector']) + \ + ',nr_sect=' + str(seg['nr_sectors']) + \ + ',readonly=' + str(not re.match('w',rw)) + ')' + xend_response = xenctl.utils.xend_control_message(cmsg) + if not xend_response['success']: + print "Error creating virtual block device" + print "Error type: " + xend_response['error_type'] + if xend_response['error_type'] == 'exception': + print "Exception type: " + xend_response['exception_type'] + print "Exception val: " + xend_response['exception_value'] + xc.domain_destroy ( dom=id ) + sys.exit() + else: + # check that setting up this VBD won't violate the sharing + # allowed by the current VBD expertise level + if xenctl.utils.vd_extents_validate(segments, + rw=='w' or rw=='rw') < 0: + xc.domain_destroy( dom = id ) + sys.exit() + + if xc.vbd_create( dom=id, vbd=virt_dev, + writeable= rw=='w' or rw=='rw' ): + print "Error creating VBD %d (writeable=%d)\n" % (virt_dev,rw) + xc.domain_destroy ( dom=id ) + sys.exit() - if xc.vbd_setextents( dom=id, - vbd=virt_dev, - extents=segments): - print "Error populating VBD vbd=%d\n" % virt_dev + if xc.vbd_setextents( dom=id, + vbd=virt_dev, + extents=segments): + print "Error populating VBD vbd=%d\n" % virt_dev + xc.domain_destroy ( dom=id ) + sys.exit() + else: # It's a block backend - notify Xend. + cmsg = 'set_block_backend(dom='+str(id)+')' + xend_response = xenctl.utils.xend_control_message(cmsg) + if not xend_response['success']: + print "Error registering network backend" + print "Error type: " + xend_response['error_type'] + if xend_response['error_type'] == 'exception': + print "Exception type: " + xend_response['exception_type'] + print "Exception val: " + xend_response['exception_value'] xc.domain_destroy ( dom=id ) sys.exit() diff --git a/tools/xend/lib/blkif.py b/tools/xend/lib/blkif.py index c0f4784e68..456e3601a0 100644 --- a/tools/xend/lib/blkif.py +++ b/tools/xend/lib/blkif.py @@ -11,6 +11,7 @@ CMSG_BLKIF_BE = 1 CMSG_BLKIF_FE = 2 CMSG_BLKIF_FE_INTERFACE_STATUS_CHANGED = 0 CMSG_BLKIF_FE_DRIVER_STATUS_CHANGED = 32 +CMSG_BLKIF_BE_DRIVER_STATUS_CHANGED = 32 CMSG_BLKIF_FE_INTERFACE_CONNECT = 33 CMSG_BLKIF_FE_INTERFACE_DISCONNECT = 34 CMSG_BLKIF_BE_CREATE = 0 @@ -22,11 +23,19 @@ CMSG_BLKIF_BE_VBD_DESTROY = 5 CMSG_BLKIF_BE_VBD_GROW = 6 CMSG_BLKIF_BE_VBD_SHRINK = 7 +BLKIF_DRIVER_STATUS_DOWN = 0 +BLKIF_DRIVER_STATUS_UP = 1 + pendmsg = None pendaddr = None +recovery = False # Is a recovery in progress? (if so we'll need to notify guests) +be_port = None # Port object for backend domain + def backend_tx_req(msg): - port = xend.main.dom0_port + port = xend.blkif.be_port + if not port: + print "BUG: attempt to transmit request to non-existant blkif driver" if port.space_to_write_request(): port.write_request(msg) port.notify() @@ -35,6 +44,26 @@ def backend_tx_req(msg): def backend_rx_req(port, msg): port.write_response(msg) + subtype = (msg.get_header())['subtype'] + print "Received blkif-be request, subtype %d" % subtype + if subtype == CMSG_BLKIF_BE_DRIVER_STATUS_CHANGED: + (status, dummy) = struct.unpack("II", msg.get_payload()) + if status == BLKIF_DRIVER_STATUS_UP: + if xend.blkif.recovery: + # Nasty hack: we count the number of VBDs we reattach so that + # we'll know when to notify the guests. Must make this better! + interface.rebuilt_so_far = 0 + interface.nr_to_rebuild = 0 + print "New blkif backend now UP, rebuilding VBDs:" + for blkif_key in interface.list.keys(): + blkif = interface.list[blkif_key] + blkif.create() + for vdev in blkif.devices.keys(): + blkif.reattach_device(vdev) + interface.nr_to_rebuild += 1 + else: + print "Unexpected block backend driver status: %d" % status + def backend_rx_rsp(port, msg): subtype = (msg.get_header())['subtype'] @@ -58,8 +87,23 @@ def backend_rx_rsp(port, msg): pdev,start_sect,nr_sect,0)) backend_tx_req(msg) elif subtype == CMSG_BLKIF_BE_VBD_GROW: - rsp = { 'success': True } - xend.main.send_management_response(rsp, xend.blkif.pendaddr) + if not xend.blkif.recovery: + rsp = { 'success': True } + xend.main.send_management_response(rsp, xend.blkif.pendaddr) + else: + interface.rebuilt_so_far += 1 + if interface.rebuilt_so_far == interface.nr_to_rebuild: + print "Rebuilt VBDs, notifying guests:" + for blkif_key in interface.list.keys(): + blkif = interface.list[blkif_key] + print " Notifying %d" % blkif.dom + msg = xend.utils.message(CMSG_BLKIF_FE, \ + CMSG_BLKIF_FE_INTERFACE_STATUS_CHANGED, 0) + msg.append_payload(struct.pack("III", 0,1,0)) + blkif.ctrlif_tx_req(xend.main.port_from_dom(blkif.dom),msg) + xend.blkif.recovery = False + print "Done notifying guests" + def backend_do_work(port): global pendmsg @@ -75,7 +119,6 @@ class interface: # Dictionary of all block-device interfaces. list = {} - # NB. 'key' is an opaque value that has no meaning in this class. def __init__(self, dom, key): self.dom = dom @@ -83,8 +126,11 @@ class interface: self.devices = {} self.pendmsg = None interface.list[key] = self + self.create() + + def create(self): msg = xend.utils.message(CMSG_BLKIF_BE, CMSG_BLKIF_BE_CREATE, 0) - msg.append_payload(struct.pack("III",dom,0,0)) + msg.append_payload(struct.pack("III",self.dom,0,0)) xend.blkif.pendaddr = xend.main.mgmt_req_addr backend_tx_req(msg) @@ -99,6 +145,12 @@ class interface: backend_tx_req(msg) return True + def reattach_device(self, vdev): + (pdev, start_sect, nr_sect, readonly) = self.devices[vdev] + msg = xend.utils.message(CMSG_BLKIF_BE, CMSG_BLKIF_BE_VBD_CREATE, 0) + msg.append_payload(struct.pack("IIHII",self.dom,0,vdev,readonly,0)) + xend.blkif.pendaddr = xend.main.mgmt_req_addr + backend_tx_req(msg) # Completely destroy this interface. def destroy(self): @@ -128,14 +180,18 @@ class interface: port.write_response(msg) subtype = (msg.get_header())['subtype'] if subtype == CMSG_BLKIF_FE_DRIVER_STATUS_CHANGED: + print "BLKIF: Domain %d says hello" % port.remote_dom msg = xend.utils.message(CMSG_BLKIF_FE, \ CMSG_BLKIF_FE_INTERFACE_STATUS_CHANGED, 0) msg.append_payload(struct.pack("III",0,1,0)) self.ctrlif_tx_req(port, msg) elif subtype == CMSG_BLKIF_FE_INTERFACE_CONNECT: + print "BLKIF: Domain %d wants to connect" % port.remote_dom (hnd,frame) = struct.unpack("IL", msg.get_payload()) xc = Xc.new() - self.evtchn = xc.evtchn_bind_interdomain(dom1=0,dom2=self.dom) + self.evtchn = xc.evtchn_bind_interdomain( \ + dom1=xend.blkif.be_port.remote_dom, \ + dom2=self.dom) msg = xend.utils.message(CMSG_BLKIF_BE, \ CMSG_BLKIF_BE_CONNECT, 0) msg.append_payload(struct.pack("IIILI",self.dom,0, \ diff --git a/tools/xend/lib/main.py b/tools/xend/lib/main.py index 65ba7f2d38..f180b5d492 100755 --- a/tools/xend/lib/main.py +++ b/tools/xend/lib/main.py @@ -72,6 +72,8 @@ def daemon_loop(): # Note that console messages don't come our way (actually, only driver # back-ends should use the DOM0 control interface). dom0_port = xend.utils.port(0) + xend.netif.be_port = dom0_port + xend.blkif_be_port = dom0_port notifier.bind(dom0_port.local_port) port_list[dom0_port.local_port] = dom0_port @@ -195,7 +197,7 @@ def daemon_loop(): con_if.ctrlif_rx_req(port, msg) elif type == CMSG_BLKIF_FE and blk_if: blk_if.ctrlif_rx_req(port, msg) - elif type == CMSG_BLKIF_BE and port == dom0_port: + elif type == CMSG_BLKIF_BE and port == xend.blkif.be_port: xend.blkif.backend_rx_req(port, msg) elif type == CMSG_NETIF_FE and net_if: net_if.ctrlif_rx_req(port, msg) @@ -209,7 +211,7 @@ def daemon_loop(): msg = port.read_response() work_done = True type = (msg.get_header())['type'] - if type == CMSG_BLKIF_BE and port == dom0_port: + if type == CMSG_BLKIF_BE and port == xend.blkif.be_port: xend.blkif.backend_rx_rsp(port, msg) elif type == CMSG_NETIF_BE and port == xend.netif.be_port: xend.netif.backend_rx_rsp(port, msg) diff --git a/tools/xend/lib/manager.py b/tools/xend/lib/manager.py index 0b301212e3..25e8e3a181 100644 --- a/tools/xend/lib/manager.py +++ b/tools/xend/lib/manager.py @@ -161,3 +161,13 @@ def set_network_backend(dom): if xend.netif.be_port: xend.netif.recovery = True xend.netif.be_port = xend.main.port_from_dom(dom) return { 'success' : True } + +## +## set_block_backend +## Authorise a domain to act as the block backend (assumes we only have one +## backend driver for now). After this call, back end "up" notifications +## for the network will only be accepted from this domain. +def set_block_backend(dom): + if xend.blkif.be_port: xend.blkif.recovery = True + xend.blkif.be_port = xend.main.port_from_dom(dom) + return { 'success' : True } diff --git a/xenolinux-2.4.26-sparse/arch/xen/drivers/blkif/backend/main.c b/xenolinux-2.4.26-sparse/arch/xen/drivers/blkif/backend/main.c index b5d406ba5e..803af976d2 100644 --- a/xenolinux-2.4.26-sparse/arch/xen/drivers/blkif/backend/main.c +++ b/xenolinux-2.4.26-sparse/arch/xen/drivers/blkif/backend/main.c @@ -487,7 +487,8 @@ static int __init init_module(void) { int i; - if ( !(start_info.flags & SIF_INITDOMAIN) ) + if ( !(start_info.flags & SIF_INITDOMAIN) + && !(start_info.flags & SIF_BLK_BE_DOMAIN) ) return 0; blkif_interface_init(); -- 2.30.2